<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta name="keywords" content="Hexo Theme Redefine">
    
    <meta name="author" content="西门啸雪">
    <!-- preconnect -->
    <link rel="preconnect" href="https://fonts.googleapis.com">
    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>

    
    <!--- Seo Part-->
    
    <link rel="canonical" href="http://aurobreeze.top/2024/06/21/python爬虫系列-beautifulsoup爬虫与线程池/"/>
    <meta name="robots" content="index,follow">
    <meta name="googlebot" content="index,follow">
    <meta name="revisit-after" content="1 days">
    
        <meta name="description" content="前言这篇文章，也是对CSDN文章的修改，使用beautifulsoup爬取网页，并使用线程池提高爬取速度。 Python代码1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950import refrom concurrent.futures import ThreadP">
<meta property="og:type" content="article">
<meta property="og:title" content="python爬虫系列:beautifulsoup爬虫与线程池">
<meta property="og:url" content="http://aurobreeze.top/2024/06/21/python%E7%88%AC%E8%99%AB%E7%B3%BB%E5%88%97-beautifulsoup%E7%88%AC%E8%99%AB%E4%B8%8E%E7%BA%BF%E7%A8%8B%E6%B1%A0/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="前言这篇文章，也是对CSDN文章的修改，使用beautifulsoup爬取网页，并使用线程池提高爬取速度。 Python代码1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950import refrom concurrent.futures import ThreadP">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-03-30.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-10-46.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-12-54.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-16-45.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-29-22.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-29-57.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-33-03.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-43-03.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-47-29.png">
<meta property="og:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_18-04-12.png">
<meta property="article:published_time" content="2024-06-21T08:39:53.000Z">
<meta property="article:modified_time" content="2024-08-11T12:56:39.296Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="爬虫">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://aurobreeze.top/blog_img/py_bs4_2/Snipaste_2024-06-21_17-03-30.png">
    
    
    <!--- Icon Part-->
    <link rel="icon" type="image/png" href="/images/redefine-favicon.svg" sizes="192x192">
    <link rel="apple-touch-icon" sizes="180x180" href="/images/redefine-favicon.svg">
    <meta name="theme-color" content="#A31F34">
    <link rel="shortcut icon" href="/images/redefine-favicon.svg">
    <!--- Page Info-->
    
    <title>
        
            python爬虫系列:beautifulsoup爬虫与线程池 -
        
        AuroBreeze
    </title>

    
<link rel="stylesheet" href="/fonts/Chillax/chillax.css">


    <!--- Inject Part-->
    

    
<link rel="stylesheet" href="/css/style.css">


    
        
<link rel="stylesheet" href="/assets/build/styles.css">

    

    
<link rel="stylesheet" href="/fonts/GeistMono/geist-mono.css">

    
<link rel="stylesheet" href="/fonts/Geist/geist.css">

    <!--- Font Part-->
    
    
    
    

    
        
<script src="/js/libs/anime.min.js"></script>

    

    <script id="hexo-configurations">
    window.config = {"hostname":"aurobreeze.top","root":"/","language":"zh-CN","path":"search.xml"};
    window.theme = {"articles":{"style":{"font_size":"16px","line_height":1.5,"image_border_radius":"14px","image_alignment":"center","image_caption":false,"link_icon":true,"title_alignment":"left","headings_top_spacing":{"h1":"3.2rem","h2":"2.4rem","h3":"1.9rem","h4":"1.6rem","h5":"1.4rem","h6":"1.3rem"}},"word_count":{"enable":true,"count":true,"min2read":true},"author_label":{"enable":true,"auto":false,"list":[]},"code_block":{"copy":true,"style":"simple","font":{"enable":false,"family":null,"url":null}},"toc":{"enable":true,"max_depth":3,"number":false,"expand":true,"init_open":true},"copyright":{"enable":true,"default":"cc_by_nc_sa"},"lazyload":true,"recommendation":{"enable":false,"title":"推荐阅读","limit":3,"mobile_limit":2,"placeholder":"/images/wallhaven-wqery6-light.webp","skip_dirs":[]}},"colors":{"primary":"#A31F34","secondary":null,"default_mode":"light"},"global":{"fonts":{"chinese":{"enable":false,"family":null,"url":null},"english":{"enable":false,"family":null,"url":null}},"content_max_width":"1000px","sidebar_width":"210px","hover":{"shadow":true,"scale":false},"scroll_progress":{"bar":false,"percentage":true},"website_counter":{"url":"https://cn.vercount.one/js","enable":true,"site_pv":false,"site_uv":false,"post_pv":true},"single_page":true,"preloader":true,"open_graph":true,"google_analytics":{"enable":false,"id":null}},"home_banner":{"enable":true,"style":"fixed","image":{"light":"/images/wallhaven-wqery6-light.webp","dark":"/images/wallhaven-wqery6-dark.webp"},"title":"AuroBreeze","subtitle":{"text":["等清风拂面,笑看人间"],"hitokoto":{"enable":false,"api":"https://v1.hitokoto.cn"},"typing_speed":80,"backing_speed":80,"starting_delay":1000,"backing_delay":1500,"loop":true,"smart_backspace":true},"text_color":{"light":"#fff","dark":"#d1d1b6"},"text_style":{"title_size":"2.8rem","subtitle_size":"1.5rem","line_height":1.2},"custom_font":{"enable":false,"family":null,"url":null},"social_links":{"enable":true,"style":"default","links":{"github":"https://github.com/AuroBreeze","instagram":null,"zhihu":null,"twitter":null,"email":"1732373074@aliyun.com"},"qrs":{"weixin":null}}},"plugins":{"feed":{"enable":false},"aplayer":{"enable":false,"type":"fixed","audios":[{"name":null,"artist":null,"url":null,"cover":null,"lrc":null}]},"mermaid":{"enable":false,"version":"9.3.0"}},"version":"2.6.4","navbar":{"auto_hide":false,"color":{"left":"#f78736","right":"#367df7","transparency":35},"width":{"home":"1200px","pages":"1000px"},"links":{"Home":{"path":"/","icon":"fa-regular fa-house"},"Archives":{"path":"/archives","icon":"fa-regular fa-archive"},"Tags":{"icon":"fa-solid fa-tags","path":"/tags/"},"Categories":{"icon":"fa-solid fa-folder","path":"/categories/"},"Friends":{"icon":"fa-solid fa-link","path":"/links/"},"About":{"icon":"fa-regular fa-user","submenus":{"Me":"/about","Github":"https://github.com/AuroBreeze","Blog":"https://aurobreeze.top"}}},"search":{"enable":true,"preload":true}},"page_templates":{"friends_column":2,"tags_style":"blur"},"home":{"sidebar":{"enable":true,"position":"left","first_item":"menu","announcement":"西门啸雪的小窝","show_on_mobile":true,"links":{"Archives":{"path":"/archives","icon":"fa-regular fa-archive"},"Tags":{"path":"/tags","icon":"fa-regular fa-tags"},"Categories":{"path":"/categories","icon":"fa-regular fa-folder"}}},"article_date_format":"auto","categories":{"enable":true,"limit":3},"tags":{"enable":true,"limit":3}},"footerStart":"2024/8/10 12:00:00"};
    window.lang_ago = {"second":"%s 秒前","minute":"%s 分钟前","hour":"%s 小时前","day":"%s 天前","week":"%s 周前","month":"%s 个月前","year":"%s 年前"};
    window.data = {"masonry":false};
  </script>
    
    <!--- Fontawesome Part-->
    
<link rel="stylesheet" href="/fontawesome/fontawesome.min.css">

    
<link rel="stylesheet" href="/fontawesome/brands.min.css">

    
<link rel="stylesheet" href="/fontawesome/solid.min.css">

    
<link rel="stylesheet" href="/fontawesome/regular.min.css">

    
    
    
    
    <script charset="UTF-8" id="LA_COLLECT" src="//sdk.51.la/js-sdk-pro.min.js"></script>
    <script>LA.init({id:"3JKUPn8RBXzO8Bzv",ck:"3JKUPn8RBXzO8Bzv"})</script>
<meta name="generator" content="Hexo 7.3.0"></head>


<body>
<div class="progress-bar-container">
    

    
        <span class="pjax-progress-bar"></span>
<!--        <span class="swup-progress-icon">-->
<!--            <i class="fa-solid fa-circle-notch fa-spin"></i>-->
<!--        </span>-->
    
</div>



    <style>
    :root {
        --preloader-background-color: #fff;
        --preloader-text-color: #000;
    }

    @media (prefers-color-scheme: dark) {
        :root {
            --preloader-background-color: #202124;
            --preloader-text-color: #fff;
        }
    }

    @media (prefers-color-scheme: light) {
        :root {
            --preloader-background-color: #fff;
            --preloader-text-color: #000;
        }
    }

    @media (max-width: 600px) {
        .ml13 {
            font-size: 2.6rem !important; /* Adjust this value as needed */
        }
    }

    .preloader {
        display: flex;
        flex-direction: column;
        gap: 1rem; /* Tailwind 'gap-4' is 1rem */
        align-items: center;
        justify-content: center;
        position: fixed;
        padding: 12px;
        top: 0;
        right: 0;
        bottom: 0;
        left: 0;
        width: 100vw;
        height: 100vh; /* 'h-screen' is 100% of the viewport height */
        background-color: var(--preloader-background-color);
        z-index: 1100; /* 'z-[1100]' sets the z-index */
        transition: opacity 0.2s ease-in-out;
    }

    .ml13 {
        font-size: 3.2rem;
        /* text-transform: uppercase; */
        color: var(--preloader-text-color);
        letter-spacing: -1px;
        font-weight: 500;
        font-family: 'Chillax-Variable', sans-serif;
        text-align: center;
    }

    .ml13 .word {
        display: inline-flex;
        flex-wrap: wrap;
        white-space: nowrap;
    }

    .ml13 .letter {
        display: inline-block;
        line-height: 1em;
    }
</style>

<div class="preloader">
    <h2 class="ml13">
        AuroBreeze
    </h2>
    <script>
        var textWrapper = document.querySelector('.ml13');
        // Split text into words
        var words = textWrapper.textContent.trim().split(' ');

        // Clear the existing content
        textWrapper.innerHTML = '';

        // Wrap each word and its letters in spans
        words.forEach(function(word) {
            var wordSpan = document.createElement('span');
            wordSpan.classList.add('word');
            wordSpan.innerHTML = word.replace(/\S/g, "<span class='letter'>$&</span>");
            textWrapper.appendChild(wordSpan);
            textWrapper.appendChild(document.createTextNode(' ')); // Add space between words
        });

        var animation = anime.timeline({loop: true})
            .add({
                targets: '.ml13 .letter',
                translateY: [40,0],
                translateZ: 0,
                opacity: [0,1],
                filter: ['blur(5px)', 'blur(0px)'], // Starting from blurred to unblurred
                easing: "easeOutExpo",
                duration: 1400,
                delay: (el, i) => 300 + 30 * i,
            }).add({
                targets: '.ml13 .letter',
                translateY: [0,-40],
                opacity: [1,0],
                filter: ['blur(0px)', 'blur(5px)'], // Ending from unblurred to blurred
                easing: "easeInExpo",
                duration: 1200,
                delay: (el, i) => 100 + 30 * i,
                complete: function() {
                    hidePreloader(); // Call hidePreloader after the animation completes
                }
            });

        let themeStatus = JSON.parse(localStorage.getItem('REDEFINE-THEME-STATUS'))?.isDark;

        // If the theme status is not found in local storage, check the preferred color scheme
        if (themeStatus === undefined || themeStatus === null) {
            if (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) {
                themeStatus = 'dark';
            } else {
                themeStatus = 'light';
            }
        }

        // Now you can use the themeStatus variable in your code
        if (themeStatus) {
            document.documentElement.style.setProperty('--preloader-background-color', '#202124');
            document.documentElement.style.setProperty('--preloader-text-color', '#fff');
        } else {
            document.documentElement.style.setProperty('--preloader-background-color', '#fff');
            document.documentElement.style.setProperty('--preloader-text-color', '#000');
        }

        window.addEventListener('load', function () {
            setTimeout(hidePreloader, 5000); // Call hidePreloader after 5000 milliseconds if not already called by animation
        });

        function hidePreloader() {
            var preloader = document.querySelector('.preloader');
            preloader.style.opacity = '0';
            setTimeout(function () {
                preloader.style.display = 'none';
            }, 200);
        }
    </script>
</div>

<main class="page-container" id="swup">

    

    <div class="main-content-container">


        <div class="main-content-header">
            <header class="navbar-container px-6 md:px-12">

    <div class="navbar-content ">
        <div class="left">
            
            <a class="logo-title" href="/">
                
                AuroBreeze
                
            </a>
        </div>

        <div class="right">
            <!-- PC -->
            <div class="desktop">
                <ul class="navbar-list">
                    
                        
                            

                            <li class="navbar-item">
                                <!-- Menu -->
                                <a class=""
                                   href="/"
                                        >
                                    <i class="fa-regular fa-house fa-fw"></i>
                                    首页
                                    
                                </a>

                                <!-- Submenu -->
                                
                            </li>
                    
                        
                            

                            <li class="navbar-item">
                                <!-- Menu -->
                                <a class=""
                                   href="/archives"
                                        >
                                    <i class="fa-regular fa-archive fa-fw"></i>
                                    归档
                                    
                                </a>

                                <!-- Submenu -->
                                
                            </li>
                    
                        
                            

                            <li class="navbar-item">
                                <!-- Menu -->
                                <a class=""
                                   href="/tags/"
                                        >
                                    <i class="fa-solid fa-tags fa-fw"></i>
                                    标签
                                    
                                </a>

                                <!-- Submenu -->
                                
                            </li>
                    
                        
                            

                            <li class="navbar-item">
                                <!-- Menu -->
                                <a class=""
                                   href="/categories/"
                                        >
                                    <i class="fa-solid fa-folder fa-fw"></i>
                                    分类
                                    
                                </a>

                                <!-- Submenu -->
                                
                            </li>
                    
                        
                            

                            <li class="navbar-item">
                                <!-- Menu -->
                                <a class=""
                                   href="/links/"
                                        >
                                    <i class="fa-solid fa-link fa-fw"></i>
                                    友情链接
                                    
                                </a>

                                <!-- Submenu -->
                                
                            </li>
                    
                        
                            

                            <li class="navbar-item">
                                <!-- Menu -->
                                <a class="has-dropdown"
                                   href="#"
                                        onClick=&#34;return false;&#34;>
                                    <i class="fa-regular fa-user fa-fw"></i>
                                    关于
                                    <i class="fa-solid fa-chevron-down fa-fw"></i>
                                </a>

                                <!-- Submenu -->
                                
                                    <ul class="sub-menu">
                                        
                                            <li>
                                                <a href="/about">
                                                    ME
                                                </a>
                                            </li>
                                        
                                            <li>
                                                <a target="_blank" rel="noopener" href="https://github.com/AuroBreeze">
                                                    GITHUB
                                                </a>
                                            </li>
                                        
                                            <li>
                                                <a href="https://aurobreeze.top">
                                                    BLOG
                                                </a>
                                            </li>
                                        
                                    </ul>
                                
                            </li>
                    
                    
                        <li class="navbar-item search search-popup-trigger">
                            <i class="fa-solid fa-magnifying-glass"></i>
                        </li>
                    
                </ul>
            </div>
            <!-- Mobile -->
            <div class="mobile">
                
                    <div class="icon-item search search-popup-trigger"><i class="fa-solid fa-magnifying-glass"></i>
                    </div>
                
                <div class="icon-item navbar-bar">
                    <div class="navbar-bar-middle"></div>
                </div>
            </div>
        </div>
    </div>

    <!-- Mobile sheet -->
    <div class="navbar-drawer h-screen w-full absolute top-0 left-0 bg-background-color flex flex-col justify-between">
        <ul class="drawer-navbar-list flex flex-col px-4 justify-center items-start">
            
                
                    

                    <li class="drawer-navbar-item text-base my-1.5 flex flex-col w-full">
                        
                        <a class="py-1.5 px-2 flex flex-row items-center justify-between gap-1 hover:!text-primary active:!text-primary text-2xl font-semibold group border-b border-border-color hover:border-primary w-full "
                           href="/"
                        >
                            <span>
                                首页
                            </span>
                            
                                <i class="fa-regular fa-house fa-sm fa-fw"></i>
                            
                        </a>
                        

                        
                    </li>
            
                
                    

                    <li class="drawer-navbar-item text-base my-1.5 flex flex-col w-full">
                        
                        <a class="py-1.5 px-2 flex flex-row items-center justify-between gap-1 hover:!text-primary active:!text-primary text-2xl font-semibold group border-b border-border-color hover:border-primary w-full "
                           href="/archives"
                        >
                            <span>
                                归档
                            </span>
                            
                                <i class="fa-regular fa-archive fa-sm fa-fw"></i>
                            
                        </a>
                        

                        
                    </li>
            
                
                    

                    <li class="drawer-navbar-item text-base my-1.5 flex flex-col w-full">
                        
                        <a class="py-1.5 px-2 flex flex-row items-center justify-between gap-1 hover:!text-primary active:!text-primary text-2xl font-semibold group border-b border-border-color hover:border-primary w-full "
                           href="/tags/"
                        >
                            <span>
                                标签
                            </span>
                            
                                <i class="fa-solid fa-tags fa-sm fa-fw"></i>
                            
                        </a>
                        

                        
                    </li>
            
                
                    

                    <li class="drawer-navbar-item text-base my-1.5 flex flex-col w-full">
                        
                        <a class="py-1.5 px-2 flex flex-row items-center justify-between gap-1 hover:!text-primary active:!text-primary text-2xl font-semibold group border-b border-border-color hover:border-primary w-full "
                           href="/categories/"
                        >
                            <span>
                                分类
                            </span>
                            
                                <i class="fa-solid fa-folder fa-sm fa-fw"></i>
                            
                        </a>
                        

                        
                    </li>
            
                
                    

                    <li class="drawer-navbar-item text-base my-1.5 flex flex-col w-full">
                        
                        <a class="py-1.5 px-2 flex flex-row items-center justify-between gap-1 hover:!text-primary active:!text-primary text-2xl font-semibold group border-b border-border-color hover:border-primary w-full "
                           href="/links/"
                        >
                            <span>
                                友情链接
                            </span>
                            
                                <i class="fa-solid fa-link fa-sm fa-fw"></i>
                            
                        </a>
                        

                        
                    </li>
            
                
                    

                    <li class="drawer-navbar-item-sub text-base my-1.5 flex flex-col w-full">
                        
                        <div class="py-1.5 px-2 flex flex-row items-center justify-between gap-1 hover:!text-primary active:!text-primary cursor-pointer text-2xl font-semibold group border-b border-border-color hover:border-primary w-full "
                             navbar-data-toggle="submenu-About"
                        >
                            <span>
                                关于
                            </span>
                            
                                <i class="fa-solid fa-chevron-right fa-sm fa-fw transition-all"></i>
                            
                        </div>
                        

                        
                            <div class="flex-col items-start px-2 py-2 hidden" data-target="submenu-About">
                                
                                    <div class="drawer-navbar-item text-base flex flex-col justify-center items-start hover:underline active:underline hover:underline-offset-1 rounded-3xl">
                                        <a class=" text-third-text-color text-xl"
                                           href="/about">ME</a>
                                    </div>
                                
                                    <div class="drawer-navbar-item text-base flex flex-col justify-center items-start hover:underline active:underline hover:underline-offset-1 rounded-3xl">
                                        <a class=" text-third-text-color text-xl"
                                           target="_blank" rel="noopener" href="https://github.com/AuroBreeze">GITHUB</a>
                                    </div>
                                
                                    <div class="drawer-navbar-item text-base flex flex-col justify-center items-start hover:underline active:underline hover:underline-offset-1 rounded-3xl">
                                        <a class=" text-third-text-color text-xl"
                                           href="https://aurobreeze.top">BLOG</a>
                                    </div>
                                
                            </div>
                        
                    </li>
            

            
            
                
                    
                    
                    
            
        </ul>

        <div class="statistics flex justify-around my-2.5">
    <a class="item tag-count-item flex flex-col justify-center items-center w-20" href="/tags">
        <div class="number text-2xl sm:text-xl text-second-text-color font-semibold">8</div>
        <div class="label text-third-text-color text-sm">标签</div>
    </a>
    <a class="item tag-count-item flex flex-col justify-center items-center w-20" href="/categories">
        <div class="number text-2xl sm:text-xl text-second-text-color font-semibold">5</div>
        <div class="label text-third-text-color text-sm">分类</div>
    </a>
    <a class="item tag-count-item flex flex-col justify-center items-center w-20" href="/archives">
        <div class="number text-2xl sm:text-xl text-second-text-color font-semibold">10</div>
        <div class="label text-third-text-color text-sm">文章</div>
    </a>
</div>
    </div>

    <div class="window-mask"></div>

</header>


        </div>

        <div class="main-content-body">

            

            <div class="main-content">

                
                    <div class="post-page-container flex relative justify-between box-border w-full h-full">
    <div class="article-content-container">

        <div class="article-title relative w-full">
            
                <div class="w-full flex items-center pt-6 justify-start">
                    <h1 class="article-title-regular text-second-text-color tracking-tight text-4xl md:text-6xl font-semibold px-2 sm:px-6 md:px-8 py-3">python爬虫系列:beautifulsoup爬虫与线程池</h1>
                </div>
            
            </div>

        
            <div class="article-header flex flex-row gap-2 items-center px-2 sm:px-6 md:px-8">
                <div class="avatar w-[46px] h-[46px] flex-shrink-0 rounded-medium border border-border-color p-[1px]">
                    <img src="http://imgos.cn/2024/07/27/66a49edd1a14f.jpg">
                </div>
                <div class="info flex flex-col justify-between">
                    <div class="author flex items-center">
                        <span class="name text-default-text-color text-lg font-semibold">西门啸雪</span>
                        
                            <span class="author-label ml-1.5 text-xs px-2 py-0.5 rounded-small text-third-text-color border border-shadow-color-1">Lv2</span>
                        
                    </div>
                    <div class="meta-info">
                        <div class="article-meta-info">
    <span class="article-date article-meta-item">
        <i class="fa-regular fa-pen-fancy"></i>&nbsp;
        <span class="desktop">2024-06-21 16:39:53</span>
        <span class="mobile">2024-06-21 16:39:53</span>
        <span class="hover-info">创建</span>
    </span>
    
        <span class="article-date article-meta-item">
            <i class="fa-regular fa-wrench"></i>&nbsp;
            <span class="desktop">2024-08-11 20:56:39</span>
            <span class="mobile">2024-08-11 20:56:39</span>
            <span class="hover-info">更新</span>
        </span>
    

    
        <span class="article-categories article-meta-item">
            <i class="fa-regular fa-folders"></i>&nbsp;
            <ul>
                
                
                    
                        
                        <li>
                            <a href="/categories/python/">python</a>&nbsp;
                        </li>
                    
                    
                
            </ul>
        </span>
    
    
        <span class="article-tags article-meta-item">
            <i class="fa-regular fa-tags"></i>&nbsp;
            <ul>
                
                    <li>
                        <a href="/tags/%E7%88%AC%E8%99%AB/">爬虫</a>&nbsp;
                    </li>
                
            </ul>
        </span>
    

    
    
        <span class="article-wordcount article-meta-item">
            <i class="fa-regular fa-typewriter"></i>&nbsp;<span>2.7k 字</span>
        </span>
    
    
        <span class="article-min2read article-meta-item">
            <i class="fa-regular fa-clock"></i>&nbsp;<span>11 分钟</span>
        </span>
    
    
        <span class="article-pv article-meta-item">
            <i class="fa-regular fa-eye"></i>&nbsp;<span id="busuanzi_value_page_pv"></span>
        </span>
    
</div>

                    </div>
                </div>
            </div>
        

        


        <div class="article-content markdown-body px-2 sm:px-6 md:px-8 pb-8">
            <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2><p>这篇文章，也是对CSDN文章的修改，使用beautifulsoup爬取网页，并使用线程池提高爬取速度。</p>
<h2 id="Python代码"><a href="#Python代码" class="headerlink" title="Python代码"></a>Python代码</h2><div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ThreadPoolExecutor <span class="comment">#导入线程池，加速爬取</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(<span class="string">&quot;webspider_book&quot;</span>):</span><br><span class="line">    os.mkdir(<span class="string">&quot;webspider_book&quot;</span>)</span><br><span class="line"></span><br><span class="line">url =<span class="string">&quot; https://www.bigee.cc/book/78647/&quot;</span></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">get_url</span>(<span class="params">url</span>):<span class="comment">#要用两次直接写个函数算了</span></span><br><span class="line">    res = requests.get(url,headers=headers)</span><br><span class="line">    soup = BeautifulSoup(res.text,<span class="string">&quot;lxml&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> soup</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">get_max_page</span>(<span class="params">url</span>): <span class="comment">#获取下载的最大页数</span></span><br><span class="line">    <span class="built_in">list</span> = get_url(url).select(<span class="string">&quot;body &gt; div.listmain &gt; dl &gt; dd a&quot;</span>)</span><br><span class="line"></span><br><span class="line">    page_finall = re.findall(<span class="string">r&quot;/book/78647/(.*).html&quot;</span>, <span class="built_in">str</span>(<span class="built_in">list</span>[-<span class="number">1</span>][<span class="string">&#x27;href&#x27;</span>]))[<span class="number">0</span>] <span class="comment">#正则匹配最后一个章节的页码</span></span><br><span class="line">    <span class="comment">#print(page_finall)</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> page_finall <span class="comment">#返回最大页数</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">get_book_content</span>(<span class="params">url</span>):<span class="comment">#下载单个页面的函数</span></span><br><span class="line"></span><br><span class="line">    soup = get_url(url)</span><br><span class="line">    name = soup.select(<span class="string">&quot;#read &gt; div.book.reader &gt; div.content &gt; h1&quot;</span>)[<span class="number">0</span>].get_text()</span><br><span class="line">    <span class="comment">#print(name)</span></span><br><span class="line">    content = soup.select(<span class="string">&quot;#chaptercontent&quot;</span>) <span class="comment">#获取章节内容</span></span><br><span class="line">    content = <span class="built_in">str</span>(content[<span class="number">0</span>].get_text()) <span class="comment">#获取章节内容</span></span><br><span class="line">    <span class="comment">#print(content)</span></span><br><span class="line"></span><br><span class="line">    f = <span class="built_in">open</span>(<span class="string">f&quot;webspider_book/<span class="subst">&#123;name&#125;</span>.txt&quot;</span>, <span class="string">&quot;w&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">    f.write(content)<span class="comment">#写入文件</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;<span class="subst">&#123;name&#125;</span>下载完成----------&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    num = <span class="built_in">int</span>(get_max_page(url=url))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;共<span class="subst">&#123;num&#125;</span>页----------&quot;</span>)</span><br><span class="line">    <span class="keyword">with</span> ThreadPoolExecutor(<span class="number">100</span>) <span class="keyword">as</span> t: <span class="comment">#线程池大小为100</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>,num+<span class="number">1</span>):</span><br><span class="line">            t.submit(get_book_content,<span class="string">f&quot;https://www.bigee.cc/book/78647/<span class="subst">&#123;i&#125;</span>.html&quot;</span>) <span class="comment">#提交任务</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;线程池执行完毕&quot;</span>)</span><br></pre></td></tr></table></figure></div>

<h2 id="正文"><a href="#正文" class="headerlink" title="正文"></a>正文</h2><h3 id="准备工作"><a href="#准备工作" class="headerlink" title="准备工作"></a>准备工作</h3><hr>
<p>我们需要下载的是<code>requests</code>和<code>beautifulsoup4</code>库，如果没有安装，请使用以下命令安装,<code>lxml</code>也需要下载。</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install requests</span><br></pre></td></tr></table></figure></div>

<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install beautifulsoup4</span><br></pre></td></tr></table></figure></div>

<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install lxml</span><br></pre></td></tr></table></figure></div>

<hr>
<p>对于线程池，我们需要导入<code>concurrent.futures</code>库，并创建一个<code>ThreadPoolExecutor</code>对象，这个就不需要下载了，这是自带的，对于<code>concurren.futures</code>的使用，我们这里只讲解一下<code>ThreadPoolExecutor</code>的使用，其他的内容大家自行探索。</p>
<hr>
<h3 id="代码-分析"><a href="#代码-分析" class="headerlink" title="代码:分析"></a>代码:分析</h3><p><strong>思路分析</strong></p>
<p>这次我们的目标是爬取目标网站的小说，下载下来并用线程池加速下载,目标网址为：</p>
<p><a class="link"   target="_blank" rel="noopener" href="https://www.bigee.cc/book/78647/" >https://www.bigee.cc/book/78647/ <i class="fa-regular fa-arrow-up-right-from-square fa-sm"></i></a></p>
<p>好了，想要爬去内容，我们需要先分析网站的结构，去看看我们要爬取的内容在哪里。</p>
<blockquote>
<p>按 <strong>F12</strong> 进入开发者工具</p>
</blockquote>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-03-30.png"
                     
                ></p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-10-46.png"
                     
                ></p>
<p>这样我们就可以看到了小说的地址到底是不是，我们可以进去看一看。</p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-12-54.png"
                     
                ></p>
<p>我们也知道，我们看到的是相对的位置，然后我们在小说内容的部分看到了他的地址</p>
<blockquote>
<p><a class="link"   target="_blank" rel="noopener" href="https://www.bigee.cc/book/78647/1.html" >https://www.bigee.cc/book/78647/1.html <i class="fa-regular fa-arrow-up-right-from-square fa-sm"></i></a></p>
</blockquote>
<p>我们看到的相对位置</p>
<blockquote>
<p>&#x2F;book&#x2F;78647&#x2F;1.html</p>
</blockquote>
<p>我们这样就察觉到他们之间不就只差了一个<code>https://www.bigee.cc</code>吗？到时候我们只需要拼接一下，就可以拿到我们想要的页面的url了。</p>
<p>我们再看其他章节的地址，我们可以看到，他们的地址都是类似的，只是最后的数字不同。</p>
<blockquote>
<p><a class="link"   target="_blank" rel="noopener" href="https://www.bigee.cc/book/78647/2.html" >https://www.bigee.cc/book/78647/2.html <i class="fa-regular fa-arrow-up-right-from-square fa-sm"></i></a></p>
</blockquote>
<blockquote>
<p><a class="link"   target="_blank" rel="noopener" href="https://www.bigee.cc/book/78647/3.html" >https://www.bigee.cc/book/78647/3.html <i class="fa-regular fa-arrow-up-right-from-square fa-sm"></i></a><br><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-16-45.png"
                     
                ></p>
</blockquote>
<p>所以如果我们想要下载所有的章节，我们只需要遍历所有的数字，然后拼接一下就可以了。</p>
<p>那我们就需要先获取到这个最大的页码，然后遍历所有的数字，再在页面中下载内容就可以了。</p>
<h3 id="代码-获取最大页码"><a href="#代码-获取最大页码" class="headerlink" title="代码:获取最大页码"></a>代码:获取最大页码</h3><p>那我们现在要做的就是获取最大的页码，我们可以用<code>requests</code>和<code>beautifulsoup4</code>来获取。</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">url =<span class="string">&quot; https://www.bigee.cc/book/78647/&quot;</span></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&quot;</span></span><br><span class="line">&#125; <span class="comment">#设置请求头,模拟浏览器</span></span><br><span class="line"></span><br><span class="line">res = requests.get(url,headers=headers).text<span class="comment">#获取网页内容</span></span><br><span class="line"></span><br><span class="line">soup = BeautifulSoup(res,<span class="string">&quot;lxml&quot;</span>)<span class="comment">#解析网页内容,加载lxml解析器</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">list</span> = soup.select(<span class="string">&quot;body &gt; div.listmain &gt; dl &gt; dd a&quot;</span>)</span><br><span class="line"></span><br><span class="line">page_finall = re.findall(<span class="string">r&quot;/book/78647/(.*).html&quot;</span>,<span class="built_in">list</span>[-<span class="number">1</span>][<span class="string">&#x27;href&#x27;</span>])</span><br><span class="line"><span class="built_in">print</span>(page_finall)</span><br></pre></td></tr></table></figure></div>

<p>这里我们使用<code>beautifulsoup</code>的<code>select</code>来获取到所有的章节的链接，也就是拿到所以我们匹配的<code>a</code>标签，这会生成一个列表。</p>
<p>对于BS4怎么去选择这个标签，我们也可以直接去浏览器进行复制，具体BS4的选择方法，大家可以自行探索。</p>
<p>下面是复制的步骤</p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-29-22.png"
                     
                ></p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-29-57.png"
                     
                ></p>
<p>但是我们复制的结果是：<code>body &gt; div.listmain &gt; dl &gt; dd:nth-child(2) &gt; a</code></p>
<p>因为我们要拿到最后一个章节的链接，而这个选择只能让我们拿到第一个章节的url，所以我们需要改一下，改为：<code>body &gt; div.listmain &gt; dl &gt; dd a</code></p>
<p>我们就能拿到这样的信息了。</p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-33-03.png"
                     
                ></p>
<p>这是一个列表，最后的章节在列表的最后一个，我们可以用<code>list[-1]</code>来获取到最后一个章节的链接。</p>
<p>我们通过<code>list[-1][&#39;href&#39;]</code>来获取到最后一个章节的链接,这是bs4的一种方法。</p>
<p>接下来我们用正则表达式来匹配这个链接，正则表达式的匹配规则是：<code>r&quot;/book/78647/(.*).html&quot;</code></p>
<blockquote>
<p>其中<code>（.*）</code>的意思是匹配任意字符，当然必须在<code>/book/78647/</code>和<code>.html</code>之间。</p>
</blockquote>
<p>对于正则的使用可以来这里学习：<a class="link"   target="_blank" rel="noopener" href="https://deerchao.cn/tutorials/regex/regex.htm" >https://deerchao.cn/tutorials/regex/regex.htm <i class="fa-regular fa-arrow-up-right-from-square fa-sm"></i></a></p>
<p>这样最后我们就拿到了最大的页码了，也就是313</p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-43-03.png"
                     
                ></p>
<hr>
<blockquote>
<p><strong>在这个网站中还有点小问题他隐藏了一部分章节的url在另一个代码块中，大家可以去看看</strong></p>
</blockquote>
<hr>
<h3 id="代码-下载"><a href="#代码-下载" class="headerlink" title="代码:下载"></a>代码:下载</h3><p>我们现在需要先写一个代码把第一页的内容都下载下来，其他的页面，除了url不一样以外其他的相同。</p>
<p>我们先进入第一页，F12进入开发者工具，分析一下。</p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_17-47-29.png"
                     
                ></p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(<span class="string">&quot;book&quot;</span>):</span><br><span class="line">    os.mkdir(<span class="string">&quot;book&quot;</span>)</span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">url = <span class="string">&quot;https://www.bigee.cc/book/78647/1.html&quot;</span></span><br><span class="line">res = requests.get(url,headers=headers).text</span><br><span class="line">soup = BeautifulSoup(res,<span class="string">&quot;lxml&quot;</span>)</span><br><span class="line"></span><br><span class="line">name = soup.select(<span class="string">&quot;#read &gt; div.book.reader &gt; div.content &gt; h1&quot;</span>)[<span class="number">0</span>].get_text() <span class="comment">#获取小说名</span></span><br><span class="line"><span class="comment">#print(name)</span></span><br><span class="line"></span><br><span class="line">content = soup.select(<span class="string">&quot;#chaptercontent&quot;</span>)<span class="comment">#获取章节内容</span></span><br><span class="line">content = <span class="built_in">str</span>(content[<span class="number">0</span>].get_text())<span class="comment">#获取章节内容</span></span><br><span class="line"><span class="built_in">print</span>(content)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">f = <span class="built_in">open</span>(<span class="string">f&quot;webspider_book/<span class="subst">&#123;name&#125;</span>.txt&quot;</span>,<span class="string">&quot;w&quot;</span>,encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">f.write(content)</span><br><span class="line"></span><br></pre></td></tr></table></figure></div>

<p><code>bs4</code>中获取标签内容的方法，是使用<code>get_text()</code> 方法，我们可以直接获取到章节的内容，但是我们用的<code>select</code>方法，返回的是一个列表，我们就需要拿到列表中的内容，当然，列表中也就只有这一个元素。因为我们选择器选择的<code>#chaptercontent</code>标签（浏览器复制的地址就是这个），所以我们直接用<code>content[0]</code>就能拿到章节的标签，然后用<code>get_text()</code>方法来获取到章节的内容。</p>
<p><img  
                     lazyload
                     src="/images/loading.svg"
                     data-src="/../blog_img/py_bs4_2/Snipaste_2024-06-21_18-04-12.png"
                     
                ></p>
<p>小说名同理，直接去网页上用浏览器直接复制地址就可以了。</p>
<p>然后我们把章节的内容写入文件，文件名就是小说名，我们用<code>f = open(f&quot;webspider_book/&#123;name&#125;.txt&quot;,&quot;w&quot;,encoding=&quot;utf-8&quot;)</code>来创建文件，然后用<code>f.write(content)</code>来写入内容。</p>
<p>这样我们就下载了第一页的内容。</p>
<hr>
<h3 id="代码-优化实现"><a href="#代码-优化实现" class="headerlink" title="代码:优化实现"></a>代码:优化实现</h3><p>我们现在已经有了下载单个页面的函数，我们可以用线程池来加速下载，我们可以用<code>ThreadPoolExecutor</code>来创建线程池，然后用<code>submit()</code>方法来提交任务。</p>
<p>简单来说，我们不需要用循环来慢慢下载网页内容了，我们直接用线程池来下载，线程池创建本来就需要循环，我们把有规律的url遍历提交给线程池。</p>
<p>正常放前面不经常用和经常用但不需要改动的数据</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ThreadPoolExecutor <span class="comment">#导入线程池，加速爬取</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(<span class="string">&quot;webspider_book&quot;</span>):</span><br><span class="line">    os.mkdir(<span class="string">&quot;webspider_book&quot;</span>)</span><br><span class="line"></span><br><span class="line">url =<span class="string">&quot; https://www.bigee.cc/book/78647/&quot;</span></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure></div>

<p>在前面的讲述中</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">res = requests.get(url,headers=headers)</span><br><span class="line">soup = BeautifulSoup(res.text,<span class="string">&quot;lxml&quot;</span>)</span><br></pre></td></tr></table></figure></div>

<p>用到了两次，所以就干脆写了个函数，这样就不用重复写了，让代码看起来干净些。</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">get_url</span>(<span class="params">url</span>):<span class="comment">#要用两次直接写个函数算了</span></span><br><span class="line">    res = requests.get(url,headers=headers)</span><br><span class="line">    soup = BeautifulSoup(res.text,<span class="string">&quot;lxml&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> soup</span><br></pre></td></tr></table></figure></div>

<p>因为我们需要实例化，所以这个函数就得把实例化的对象<code>soup</code>返回，这样才可以使用。</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">get_max_page</span>(<span class="params">url</span>): <span class="comment">#获取下载的最大页数</span></span><br><span class="line">    <span class="built_in">list</span> = get_url(url).select(<span class="string">&quot;body &gt; div.listmain &gt; dl &gt; dd a&quot;</span>)</span><br><span class="line"></span><br><span class="line">    page_finall = re.findall(<span class="string">r&quot;/book/78647/(.*).html&quot;</span>, <span class="built_in">str</span>(<span class="built_in">list</span>[-<span class="number">1</span>][<span class="string">&#x27;href&#x27;</span>]))[<span class="number">0</span>] <span class="comment">#正则匹配最后一个章节的页码</span></span><br><span class="line">    <span class="comment">#print(page_finall)</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> page_finall <span class="comment">#返回最大页数</span></span><br><span class="line"></span><br></pre></td></tr></table></figure></div>

<p>这个函数就是获取最大的页码,但是他也需要先实例化<code>soup</code>对象，所以我们就把<code>get_url()</code>函数也写进来，让他实例化并返回<code>soup</code>对象，最后正则获取到最后一个章节的url。</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">get_book_content</span>(<span class="params">url</span>):<span class="comment">#下载单个页面的函数</span></span><br><span class="line"></span><br><span class="line">    soup = get_url(url)</span><br><span class="line">    name = soup.select(<span class="string">&quot;#read &gt; div.book.reader &gt; div.content &gt; h1&quot;</span>)[<span class="number">0</span>].get_text()</span><br><span class="line">    <span class="comment">#print(name)</span></span><br><span class="line">    content = soup.select(<span class="string">&quot;#chaptercontent&quot;</span>) <span class="comment">#获取章节内容</span></span><br><span class="line">    content = <span class="built_in">str</span>(content[<span class="number">0</span>].get_text()) <span class="comment">#获取章节内容</span></span><br><span class="line">    <span class="comment">#print(content)</span></span><br><span class="line"></span><br><span class="line">    f = <span class="built_in">open</span>(<span class="string">f&quot;webspider_book/<span class="subst">&#123;name&#125;</span>.txt&quot;</span>, <span class="string">&quot;w&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">    f.write(content)<span class="comment">#写入文件</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;<span class="subst">&#123;name&#125;</span>下载完成----------&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure></div>

<p>这个函数是下载对应页面的内容，他也需要先实例化<code>soup</code>对象，所以我们继续用我们刚开始写的<code>get_url()</code>函数来实例化，然后用<code>select()</code>方法来获取到小说名，然后用<code>get_text()</code>方法来获取到小说名和章节内容。</p>
<div class="highlight-container" data-rel="Python"><figure class="iseeu highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    num = <span class="built_in">int</span>(get_max_page(url=url))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;共<span class="subst">&#123;num&#125;</span>页----------&quot;</span>)</span><br><span class="line">    <span class="keyword">with</span> ThreadPoolExecutor(<span class="number">100</span>) <span class="keyword">as</span> t: <span class="comment">#线程池大小为100</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>,num+<span class="number">1</span>):</span><br><span class="line">            t.submit(get_book_content,<span class="string">f&quot;https://www.bigee.cc/book/78647/<span class="subst">&#123;i&#125;</span>.html&quot;</span>) <span class="comment">#提交任务</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;线程池执行完毕&quot;</span>)</span><br></pre></td></tr></table></figure></div>

<p>在这里我们先运行<code>get_max_page()</code>函数来获取最大的页码，url我们在上面已经先声明了，然后我们用<code>ThreadPoolExecutor</code>来创建线程池，线程池大小为100 **(可以自己调整)**，然后用<code>for</code>循环来遍历所有的页码，然后用<code>submit()</code>方法来提交任务，任务就是<code>get_book_content()</code>函数，我们把页码和url拼接起来，然后提交给线程池,这样我们就能很快的把小说爬完了。</p>
<hr>
<h1 id="如果你感觉这篇文章不错，而且财力充足的话，希望可以用爱发电支持一下作者，感谢支持！"><a href="#如果你感觉这篇文章不错，而且财力充足的话，希望可以用爱发电支持一下作者，感谢支持！" class="headerlink" title="如果你感觉这篇文章不错，而且财力充足的话，希望可以用爱发电支持一下作者，感谢支持！"></a>如果你感觉这篇文章不错，而且财力充足的话，希望可以用爱发电支持一下作者，感谢支持！</h1><blockquote>
<p>爱发电地址:<a class="link"   target="_blank" rel="noopener" href="https://afdian.com/a/AuroraBreeze" >https://afdian.com/a/AuroraBreeze <i class="fa-regular fa-arrow-up-right-from-square fa-sm"></i></a></p>
</blockquote>

        </div>

        
            <div class="post-copyright-info w-full my-8 px-2 sm:px-6 md:px-8">
                <div class="article-copyright-info-container">
    <ul>
        <li><strong>标题:</strong> python爬虫系列:beautifulsoup爬虫与线程池</li>
        <li><strong>作者:</strong> 西门啸雪</li>
        <li><strong>创建于
                :</strong> 2024-06-21 16:39:53</li>
        
            <li>
                <strong>更新于
                    :</strong> 2024-08-11 20:56:39
            </li>
        
        <li>
            <strong>链接:</strong> http://aurobreeze.top/2024/06/21/python爬虫系列-beautifulsoup爬虫与线程池/
        </li>
        <li>
            <strong>
                版权声明:
            </strong>
            

            
                本文章采用 <a class="license" target="_blank" rel="noopener" href="https://creativecommons.org/licenses/by-nc-sa/4.0">CC BY-NC-SA 4.0</a> 进行许可。
            
        </li>
    </ul>
</div>

            </div>
        

        
            <ul class="post-tags-box text-lg mt-1.5 flex-wrap justify-center flex md:hidden">
                
                    <li class="tag-item mx-0.5">
                        <a href="/tags/%E7%88%AC%E8%99%AB/">#爬虫</a>&nbsp;
                    </li>
                
            </ul>
        

        

        
            <div class="article-nav my-8 flex justify-between items-center px-2 sm:px-6 md:px-8">
                
                    <div class="article-prev border-border-color shadow-redefine-flat shadow-shadow-color-2 rounded-medium px-4 py-2 hover:shadow-redefine-flat-hover hover:shadow-shadow-color-2">
                        <a class="prev"
                        rel="prev"
                        href="/2024/07/08/pywifi%E7%A0%B4%E8%A7%A3WiFi%E5%AF%86%E7%A0%81%E7%A4%BA%E4%BE%8B/"
                        >
                            <span class="left arrow-icon flex justify-center items-center">
                                <i class="fa-solid fa-chevron-left"></i>
                            </span>
                            <span class="title flex justify-center items-center">
                                <span class="post-nav-title-item">pywifi破解WiFi密码示例</span>
                                <span class="post-nav-item">上一篇</span>
                            </span>
                        </a>
                    </div>
                
                
                    <div class="article-next border-border-color shadow-redefine-flat shadow-shadow-color-2 rounded-medium px-4 py-2 hover:shadow-redefine-flat-hover hover:shadow-shadow-color-2">
                        <a class="next"
                        rel="next"
                        href="/2024/06/21/kaze%E9%85%8D%E7%BD%AE%E5%BB%BA%E8%AE%AE/"
                        >
                            <span class="title flex justify-center items-center">
                                <span class="post-nav-title-item">kaze主题配置建议</span>
                                <span class="post-nav-item">下一篇</span>
                            </span>
                            <span class="right arrow-icon flex justify-center items-center">
                                <i class="fa-solid fa-chevron-right"></i>
                            </span>
                        </a>
                    </div>
                
            </div>
        


        
            <div class="comment-container px-2 sm:px-6 md:px-8 pb-8">
                <div class="comments-container mt-10 w-full ">
    <div id="comment-anchor" class="w-full h-2.5"></div>
    <div class="comment-area-title w-full my-1.5 md:my-2.5 text-xl md:text-3xl font-bold">
        评论
    </div>
    

        
            
    <div id="giscus-container"></div>
    <script data-swup-reload-script defer>
        async function loadGiscus() {
            const giscusConfig = {
                'src': 'https://giscus.app/client.js',
                'data-repo': 'ximenxiaoxue/new_blog_comment',
                'data-repo-id': 'R_kgDOMiDymw# Github repository id',
                'data-category': 'Announcements',
                'data-category-id': 'DIC_kwDOMiDym84Chisb',
                'data-mapping': 'title',
                'data-strict': '0',
                'data-reactions-enabled': '1',
                'data-emit-metadata': '1',
                'data-theme': 'preferred_color_scheme',
                'data-lang': 'zh-CN',
                'data-input-position': 'bottom',
                'data-loading': 'not-lazy',
                'crossorigin': 'anonymous',
                'async': true
            }
            const giscusScript = document.createElement('script');
            for (const key in giscusConfig) {
                giscusScript.setAttribute(key, giscusConfig[key]);
            }
            document.getElementById('giscus-container').appendChild(giscusScript);
        }
        if ('true') {
            let loadGiscusTimeout = setTimeout(() => {
                loadGiscus();
                clearTimeout(loadGiscusTimeout);
            }, 1000);
        } else {
            document.addEventListener('DOMContentLoaded', loadGiscus);
        }
    </script>


        
        
    
</div>

            </div>
        
    </div>

    
        <div class="toc-content-container">
            <div class="post-toc-wrap">
    <div class="post-toc">
        <div class="toc-title">此页目录</div>
        <div class="page-title">python爬虫系列:beautifulsoup爬虫与线程池</div>
        <ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%89%8D%E8%A8%80"><span class="nav-text">前言</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Python%E4%BB%A3%E7%A0%81"><span class="nav-text">Python代码</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E6%AD%A3%E6%96%87"><span class="nav-text">正文</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%87%86%E5%A4%87%E5%B7%A5%E4%BD%9C"><span class="nav-text">准备工作</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BB%A3%E7%A0%81-%E5%88%86%E6%9E%90"><span class="nav-text">代码:分析</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BB%A3%E7%A0%81-%E8%8E%B7%E5%8F%96%E6%9C%80%E5%A4%A7%E9%A1%B5%E7%A0%81"><span class="nav-text">代码:获取最大页码</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BB%A3%E7%A0%81-%E4%B8%8B%E8%BD%BD"><span class="nav-text">代码:下载</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BB%A3%E7%A0%81-%E4%BC%98%E5%8C%96%E5%AE%9E%E7%8E%B0"><span class="nav-text">代码:优化实现</span></a></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#%E5%A6%82%E6%9E%9C%E4%BD%A0%E6%84%9F%E8%A7%89%E8%BF%99%E7%AF%87%E6%96%87%E7%AB%A0%E4%B8%8D%E9%94%99%EF%BC%8C%E8%80%8C%E4%B8%94%E8%B4%A2%E5%8A%9B%E5%85%85%E8%B6%B3%E7%9A%84%E8%AF%9D%EF%BC%8C%E5%B8%8C%E6%9C%9B%E5%8F%AF%E4%BB%A5%E7%94%A8%E7%88%B1%E5%8F%91%E7%94%B5%E6%94%AF%E6%8C%81%E4%B8%80%E4%B8%8B%E4%BD%9C%E8%80%85%EF%BC%8C%E6%84%9F%E8%B0%A2%E6%94%AF%E6%8C%81%EF%BC%81"><span class="nav-text">如果你感觉这篇文章不错，而且财力充足的话，希望可以用爱发电支持一下作者，感谢支持！</span></a>

    </div>
</div>
        </div>
    
</div>



                

            </div>

            

        </div>

        <div class="main-content-footer">
            <footer class="footer mt-5 py-5 h-auto text-base text-third-text-color relative border-t-2 border-t-border-color">
    <div class="info-container py-3 text-center">
        
        <div class="text-center">
            &copy;
            
              <span>2024</span>
              -
            
            2024&nbsp;&nbsp;<i class="fa-solid fa-heart fa-beat" style="--fa-animation-duration: 0.5s; color: #f54545"></i>&nbsp;&nbsp;<a href="/">西门啸雪</a>
            
                
                <p class="post-count space-x-0.5">
                    <span>
                        共 10 篇文章
                    </span>
                    
                        <span>
                            共 27.2k 字
                        </span>
                    
                </p>
            
        </div>
        
            <script data-swup-reload-script src="https://cn.vercount.one/js"></script>
            <div class="relative text-center lg:absolute lg:right-[20px] lg:top-1/2 lg:-translate-y-1/2 lg:text-right">
                
                
            </div>
        
        <div class="relative text-center lg:absolute lg:left-[20px] lg:top-1/2 lg:-translate-y-1/2 lg:text-left">
            <span class="lg:block text-sm">由 <?xml version="1.0" encoding="utf-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg class="relative top-[2px] inline-block align-baseline" version="1.1" id="圖層_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="1rem" height="1rem" viewBox="0 0 512 512" enable-background="new 0 0 512 512" xml:space="preserve"><path fill="#0E83CD" d="M256.4,25.8l-200,115.5L56,371.5l199.6,114.7l200-115.5l0.4-230.2L256.4,25.8z M349,354.6l-18.4,10.7l-18.6-11V275H200v79.6l-18.4,10.7l-18.6-11v-197l18.5-10.6l18.5,10.8V237h112v-79.6l18.5-10.6l18.5,10.8V354.6z"/></svg><a target="_blank" class="text-base" href="https://hexo.io">Hexo</a> 驱动</span>
            <span class="text-sm lg:block">主题&nbsp;<a class="text-base" target="_blank" href="https://github.com/EvanNotFound/hexo-theme-redefine">Redefine v2.6.4</a></span>
        </div>
        
            <div class="icp-info my-1"><a target="_blank" rel="nofollow" href="
                
                    https://beian.miit.gov.cn/
                
                ">鲁ICP备2024112337号-1</a></div>
        
        
            <div>
                博客已运行 <span class="odometer" id="runtime_days" ></span> 天 <span class="odometer" id="runtime_hours"></span> 小时 <span class="odometer" id="runtime_minutes"></span> 分钟 <span class="odometer" id="runtime_seconds"></span> 秒
            </div>
        
        
            <script data-swup-reload-script>
                try {
                    function odometer_init() {
                    const elements = document.querySelectorAll('.odometer');
                    elements.forEach(el => {
                        new Odometer({
                            el,
                            format: '( ddd).dd',
                            duration: 200
                        });
                    });
                    }
                    odometer_init();
                } catch (error) {}
            </script>
        
        
        
    </div>  
</footer>
        </div>
    </div>

    
        <div class="post-tools">
            <div class="post-tools-container">
    <ul class="article-tools-list">
        <!-- TOC aside toggle -->
        
            <li class="right-bottom-tools page-aside-toggle">
                <i class="fa-regular fa-outdent"></i>
            </li>
        

        <!-- go comment -->
        
            <li class="go-comment">
                <i class="fa-regular fa-comments"></i>
            </li>
        
    </ul>
</div>

        </div>
    

    <div class="right-side-tools-container">
        <div class="side-tools-container">
    <ul class="hidden-tools-list">
        <li class="right-bottom-tools tool-font-adjust-plus flex justify-center items-center">
            <i class="fa-regular fa-magnifying-glass-plus"></i>
        </li>

        <li class="right-bottom-tools tool-font-adjust-minus flex justify-center items-center">
            <i class="fa-regular fa-magnifying-glass-minus"></i>
        </li>

        <li class="right-bottom-tools tool-dark-light-toggle flex justify-center items-center">
            <i class="fa-regular fa-moon"></i>
        </li>

        <!-- rss -->
        

        

        <li class="right-bottom-tools tool-scroll-to-bottom flex justify-center items-center">
            <i class="fa-regular fa-arrow-down"></i>
        </li>
    </ul>

    <ul class="visible-tools-list">
        <li class="right-bottom-tools toggle-tools-list flex justify-center items-center">
            <i class="fa-regular fa-cog fa-spin"></i>
        </li>
        
            <li class="right-bottom-tools tool-scroll-to-top flex justify-center items-center">
                <i class="arrow-up fas fa-arrow-up"></i>
                <span class="percent"></span>
            </li>
        
        
    </ul>
</div>

    </div>

    <div class="image-viewer-container">
    <img src="">
</div>


    
        <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
          <span class="search-input-field-pre">
            <i class="fa-solid fa-keyboard"></i>
          </span>
            <div class="search-input-container">
                <input autocomplete="off"
                       autocorrect="off"
                       autocapitalize="off"
                       placeholder="搜索..."
                       spellcheck="false"
                       type="search"
                       class="search-input"
                >
            </div>
            <span class="popup-btn-close">
                <i class="fa-solid fa-times"></i>
            </span>
        </div>
        <div id="search-result">
            <div id="no-result">
                <i class="fa-solid fa-spinner fa-spin-pulse fa-5x fa-fw"></i>
            </div>
        </div>
    </div>
</div>

    

</main>


    
<script src="/js/libs/Swup.min.js"></script>

<script src="/js/libs/SwupSlideTheme.min.js"></script>

<script src="/js/libs/SwupScriptsPlugin.min.js"></script>

<script src="/js/libs/SwupProgressPlugin.min.js"></script>

<script src="/js/libs/SwupScrollPlugin.min.js"></script>

<script src="/js/libs/SwupPreloadPlugin.min.js"></script>

<script>
    const swup = new Swup({
        plugins: [
            new SwupScriptsPlugin({
                optin: true,
            }),
            new SwupProgressPlugin(),
            new SwupScrollPlugin({
                offset: 80,
            }),
            new SwupSlideTheme({
                mainElement: ".main-content-body",
            }),
            new SwupPreloadPlugin(),
        ],
        containers: ["#swup"],
    });
</script>







<script src="/js/tools/imageViewer.js" type="module"></script>

<script src="/js/utils.js" type="module"></script>

<script src="/js/main.js" type="module"></script>

<script src="/js/layouts/navbarShrink.js" type="module"></script>

<script src="/js/tools/scrollTopBottom.js" type="module"></script>

<script src="/js/tools/lightDarkSwitch.js" type="module"></script>

<script src="/js/layouts/categoryList.js" type="module"></script>



    
<script src="/js/tools/localSearch.js" type="module"></script>




    
<script src="/js/tools/codeBlock.js" type="module"></script>




    
<script src="/js/layouts/lazyload.js" type="module"></script>




    
<script src="/js/tools/runtime.js"></script>

    
<script src="/js/libs/odometer.min.js"></script>

    
<link rel="stylesheet" href="/assets/odometer-theme-minimal.css">




  
<script src="/js/libs/Typed.min.js"></script>

  
<script src="/js/plugins/typed.js" type="module"></script>









<div class="post-scripts" data-swup-reload-script>
    
        
<script src="/js/tools/tocToggle.js" type="module"></script>

<script src="/js/layouts/toc.js" type="module"></script>

<script src="/js/plugins/tabs.js" type="module"></script>

    
</div>


</body>
</html>
